package com.kd.crawler.parser;

import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.log4j.Logger;
import org.dom4j.Element;

import com.alibaba.fastjson.JSON;
import com.kd.crawler.common.utils.StringUtils;
import com.kd.crawler.common.utils.UrlUtils;
import com.kd.crawler.common.utils.XMLParser;
import com.kd.crawler.entity.CrawlerEntry;
import com.kd.crawler.entity.Document;
import com.kd.crawler.parser.parseres.JsoupParser;
import com.kd.crawler.parser.parseres.Parser;
import com.kd.crawler.parser.parseres.RegexParser;
import com.kd.crawler.parser.url.UrlParser;
import com.kdtech.analyse.AnalyseNews;
import com.kdtech.analyse.cache.AnalyseClassCache;
import com.kdtech.entity.crawler.UrlMeta;
import com.kdtech.entity.data.NewsMeta;
import com.kdtech.utils.DateUtils;

/**
 * this class is safe thread, and it's work is that parse a html of crawl result
 * into a bean, eg: BasicMeta
 * 
 * @author Administrator
 *
 */
public class HtmlParserImpl extends HtmlParser {

	private static final Logger logger = Logger.getLogger(HtmlParserImpl.class);

	public String parserName = null;

	private Map<String, List<Parser>> parseMap = new HashMap<String, List<Parser>>();

	private List<String> contentRegexList = null;

	public HtmlParserImpl(String parserName) {
		this.parserName = parserName;
		init();
	}

	private void init() {
		String parserPath = getParserPath(parserName);
		boolean isExisted = exists(parserPath);
		if (!isExisted) {
			return;
		}
		contentRegexList = getRegex(parserPath);
		List<Element> list = XMLParser.getChildrenElements(parserPath, "");
		for (Element item : list) {
			List<Element> childrenElement = item.elements();
			List<Parser> parserList = new ArrayList<Parser>();
			for (Element childrenItem : childrenElement) {
				if (childrenItem.getName().toLowerCase().equals("regex")) {
					String regex = childrenItem.getTextTrim();
					String group = childrenItem.attribute("group") == null ? "0"
							: childrenItem.attribute("group").getValue();
					Parser parser = new RegexParser(regex, Integer.valueOf(group));
					parserList.add(parser);
				} else if (childrenItem.getName().toLowerCase().equals("jsoup")) {
					String jsoup = childrenItem.getTextTrim();
					Parser parser = new JsoupParser(jsoup);
					parserList.add(parser);
				}
			}
			parseMap.put(item.getName().toLowerCase(), parserList);
		}

	}

	/**
	 * get all of inner links that are detailed page from html of crawler
	 * result.
	 * 
	 * @param cr
	 * @return
	 */
	public List<String> getDetailedInnerLinks(CrawlerEntry cr) {
		List<String> urllist = UrlParser.getInnerLinks(cr.getHtml(), cr.getUrl());
		List<String> detailedInnerLinks = new ArrayList<String>();
		AnalyseNews analyser = null;
		if (contentRegexList!=null&&contentRegexList.size()>0) {
			for(String url:urllist){
				if (UrlUtils.isDetailed(url, contentRegexList)) {
					detailedInnerLinks.add(url);
				}
			}
		} else {
			logger.info("the config file[" + parserName + ".xml] is not existed, please config it");
			// get an analyser from AnalyseClassCache
			analyser = AnalyseClassCache.getAnalyseNews(parserName);
			if (null == analyser) {
				logger.error("[analyser]url: " + cr.getUrl() + " --->analyse class[" + parserName + "] is not existed");
				return detailedInnerLinks;
			}else{
				for(String url:urllist){
					if (analyser.isDetailPage(url)) {
						detailedInnerLinks.add(url);
					}
				}
			}
		}
		
		return detailedInnerLinks;
	}

	public Document parse(CrawlerEntry cr) {
		if (null == cr) {
			return null;
		}
		String url = cr.getUrl();
		String parserName = cr.getParserName();
		String parserPath = getParserPath(parserName);
		// here, giving priority to use new
		if (!exists(parserPath)) {
			logger.info("the config file[" + parserName + ".xml] is not existed, please config it");
			Document basicMeta=null;
			try{
				basicMeta = parseNews(cr);
			}catch(Exception e){
				logger.info("解释报错["+parserName+"] url="+url,e);
			}
			if(basicMeta!=null){
				basicMeta.setCrawlTime(cr.getCrawlTime());
				logFor(url, basicMeta.getTitle(), "title", parserName,cr);
				logFor(url, basicMeta.getDate(), "date", parserName,cr);
				//logFor(url, basicMeta.getAuthor(), "author", parserName,cr);
				logFor(url, basicMeta.getContent(), "content", parserName,cr);
				if(basicMeta==null||basicMeta.getContent()==null||basicMeta.getTitle()==null||basicMeta.getDate()==null||basicMeta.getDate()<=0){
					cr.setHtml(null);
					logger.error("解释详细页内容不正常("+cr.getStatusCode()+"):"+JSON.toJSONString(cr));
				}
			}else{
				logger.error("解释详细页内容不正常("+cr.getStatusCode()+"):"+JSON.toJSONString(cr));
			}			
			return basicMeta;
		}
		
		String title =parse("title",cr.getHtml());
		String author =parse("author",cr.getHtml());
		String content =parse("content",cr.getHtml());
		String strDate =parse("date",cr.getHtml());
		Long date = DateUtils.matchDate(strDate);
		if (null == date) {
			date = DateUtils.matchDate(cr.getUrl());
		}		
	
		// log for parse result is empty or null.
		// warn: add more jsoup or regex to relative attributes: title, author,
		// date and content.
		logFor(url, title, "title", parserPath,cr);
		logFor(url, date, "date", parserPath,cr);
		//logFor(url, author, "author", parserPath,cr);
		logFor(url, content, "content", parserPath,cr);
		if(title==null||content==null||strDate==null){
			cr.setHtml(null);
			logger.error("解释详细页内容不正常("+cr.getStatusCode()+"):"+JSON.toJSONString(cr));
		}
		Document meta = new Document();
		meta.setUrl(url);
		meta.setTitle(StringUtils.trimSpace(title));
		meta.setAuthor(author);
		meta.setContent(content);
		meta.setDate(date);
		meta.setSource(cr.getJobID());
		meta.setType(cr.getTypeId());
		meta.setJobId(cr.getJobID());
		meta.setEntryID(cr.getEntryId());
		meta.setCrawlTime(cr.getCrawlTime());		
		return meta;
	}
	
	private String parse(String propertyName,String html){
		List<Parser> parserList=parseMap.get(propertyName);
		for(Parser parser:parserList){
			String result=parser.parse(html);
			if(result!=null){
				return result;
			}
		}
		return html;
	}

	/**
	 * use analyse class to parse by analyserName
	 * 
	 * @return
	 */
	public Document parseNews(CrawlerEntry cr) {
		String analyserName = cr.getParserName();
		// from analyze class
		AnalyseNews analyser = AnalyseClassCache.getAnalyseNews(analyserName);
		if (null == analyser) {
			logger.error("[analyser]url: " + cr.getUrl() + " --->analyse class[" + analyserName + "] is not existed");
			return null;
		}
		UrlMeta urlMeta = new UrlMeta();
		urlMeta.setUrl(cr.getUrl());
		urlMeta.setHtml(cr.getHtml());
		NewsMeta oldMeta = analyser.parserHtml(urlMeta);
		if (null == oldMeta) {
			return null;
		}
		// change old version meta into the current new meta.
		Document basicMeta = new Document();
		basicMeta.setUrl(cr.getUrl());
		basicMeta.setTitle(oldMeta.getTitle());
		basicMeta.setDate(oldMeta.getDate());
		basicMeta.setAuthor(oldMeta.getAuthor());
		basicMeta.setAddressValue(oldMeta.getAddressValue());
		basicMeta.setContent(oldMeta.getContent());
		basicMeta.setClickNum(oldMeta.getClickNum());
		basicMeta.setCommentNum(oldMeta.getCommentNum());
		basicMeta.setType(cr.getTypeId());
		basicMeta.setJobId(cr.getJobID());
		basicMeta.setSource(cr.getJobID());
		basicMeta.setEntryID(cr.getEntryId());
		basicMeta.setLevel(cr.getLevel());
		basicMeta.setAddressValue(cr.getRegionId());
		return basicMeta;
	}

	public static void main(String[] args) {
		String analyserName = "_010lfNewsAnalyse";
		URL parserUrl = HtmlParserImpl.class.getClassLoader().getResource(analyserName + ".xml");
		String path = parserUrl.getPath();
		System.out.println(path);
		// Class.forName(analyserName);
	}

}
